# !/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2024/9/12 09:57
# @Author  : 王凯
# @File    : cl_clean.py
# @Project : scrapy_spider

from apps.data_stats.data_stats.clean import BaseClean


class CLClean(BaseClean):

    def run_gy(self):

        def process_data(df_pro):
            item = {}
            for j in df_pro.to_dict("records"):
                if j["header"] == "同比增长":
                    item["production_yoy_growthrate"] = j["num"] / 100
                elif j["header"] == "累计增长":
                    item["production_sum_yoy_growthrate"] = j["num"] / 100
                elif j["header"] == "当期值":
                    item["production_current_value"] = j["num"]
                    item["unit"] = j["unit"]
                elif j["header"] == "累计值":
                    item["production_sum"] = j["num"]
                    item["unit"] = j["unit"]
                item["industry_name"] = j["industry_name"]
                item["time"] = j["date_string"]
                item["type"] = "工业"
            return item

        df = self.get_data_from_db(cate_1="工业", cate_2="工业主要产品产量", condition=" and area = '全国' ")
        df["industry_name"] = df["cate_3"]
        df["header"] = df["tag_name"].map(lambda x: x.split("_")[1])

        datas = list(df.groupby(["industry_name", "date_string"]).apply(process_data).to_dict().values())

        batch_size = 1000
        for i in range(0, len(datas), batch_size):
            self.to_db.add_batch_smart("net_industrialgoods_production_data", list(datas[i:i + batch_size]), update_columns=list(datas[0].keys()))

        return datas

    def run_ny(self):

        def process_data(df_pro):
            item = {}
            for j in df_pro.to_dict("records"):
                if j["header"] == "同比增长":
                    item["production_yoy_growthrate"] = j["num"] / 100
                elif j["header"] == "累计增长":
                    item["production_sum_yoy_growthrate"] = j["num"] / 100
                elif j["header"] == "当期值":
                    item["production_current_value"] = j["num"]
                    item["unit"] = j["unit"]
                elif j["header"] == "累计值":
                    item["production_sum"] = j["num"]
                    item["unit"] = j["unit"]
                item["industry_name"] = j["industry_name"]
                item["time"] = j["date_string"]
                item["type"] = "能源"
            return item

        df = self.get_data_from_db(cate_1="能源", cate_2="能源主要产品产量", condition=" and area = '全国' ")
        df["industry_name"] = df["cate_3"]
        df["header"] = df["tag_name"].map(lambda x: x.split("_")[1])

        datas = list(df.groupby(["industry_name", "date_string"]).apply(process_data).to_dict().values())

        batch_size = 1000
        for i in range(0, len(datas), batch_size):
            self.to_db.add_batch_smart("net_industrialgoods_production_data", list(datas[i:i + batch_size]))

        return datas

    def run(self):
        self.run_gy()
        self.run_ny()


if __name__ == "__main__":
    CLClean().run()
